{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tensor_homework_senior_units_count.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/henrygas/tensorflow_2_learn/blob/master/tensor_homework_senior_units_count.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PnqiqI8b2Zso",
        "colab_type": "code",
        "outputId": "89d800e1-6f8a-47a4-9a16-2a06e9e5977b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "\n",
        "print(tf.__version__)\n",
        "print(np.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.15.0-rc3\n",
            "1.16.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tlwJ-ubOYFOV",
        "colab_type": "code",
        "outputId": "0e5fc2b4-dfc9-4826-e17e-b6643adc16a8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 613
        }
      },
      "source": [
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "mnist = input_data.read_data_sets(\"./\")\n",
        "\n",
        "print(mnist.train.images.shape)\n",
        "print(mnist.train.labels.shape)\n",
        "\n",
        "print(mnist.validation.images.shape)\n",
        "print(mnist.validation.labels.shape)\n",
        "\n",
        "print(mnist.test.images.shape)\n",
        "print(mnist.test.labels.shape)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-2-e83e5b30b437>:6: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use urllib or similar directly.\n",
            "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting ./train-images-idx3-ubyte.gz\n",
            "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting ./train-labels-idx1-ubyte.gz\n",
            "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
            "Extracting ./t10k-images-idx3-ubyte.gz\n",
            "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
            "Extracting ./t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "(55000, 784)\n",
            "(55000,)\n",
            "(5000, 784)\n",
            "(5000,)\n",
            "(10000, 784)\n",
            "(10000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbrRlvQzv0TS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x = tf.placeholder(\"float\", [None, 784])\n",
        "y = tf.placeholder(\"int64\", [None])\n",
        "learning_rate = tf.placeholder(\"float\")\n",
        "\n",
        "L1_units_count = 500\n",
        "W_1 = tf.Variable(tf.truncated_normal([784, L1_units_count], stddev=0.1))\n",
        "b_1 = tf.Variable(tf.truncated_normal([L1_units_count], stddev=0.1))\n",
        "logits_1 = tf.matmul(x, W_1) + b_1\n",
        "output_1 = tf.nn.relu(logits_1)\n",
        "\n",
        "L2_units_count = 10\n",
        "W_2 = tf.Variable(tf.truncated_normal([L1_units_count, L2_units_count], stddev=0.1))\n",
        "b_2 = tf.Variable(tf.truncated_normal([L2_units_count], stddev=0.1))\n",
        "logits_2 = tf.matmul(output_1, W_2) + b_2\n",
        "\n",
        "logits = logits_2\n",
        "\n",
        "cross_entropy_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
        "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy_loss)\n",
        "\n",
        "pred = tf.nn.softmax(logits)\n",
        "correct_pred = tf.equal(tf.argmax(pred, 1), y)\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
        "\n",
        "batch_size = 32\n",
        "learning_rate_value = 0.5\n",
        "max_train_step = 1000\n",
        "saver = tf.train.Saver()\n",
        "\n",
        "acc_train_max = 0\n",
        "acc_validation_max = 0\n",
        "acc_test_max = 0\n",
        "\n",
        "test_times = 30\n",
        "while (test_times > 0):\n",
        "  test_times -= 1\n",
        "  with tf.Session() as ses:\n",
        "    ses.run(tf.global_variables_initializer())\n",
        "    for i in range(max_train_step):\n",
        "      xs, ys = mnist.train.next_batch(batch_size)\n",
        "      _, loss, W_1_value, b_1_value, W_2_value, b_2_value = ses.run([optimizer, cross_entropy_loss, W_1, b_1, W_2, b_2], feed_dict={\n",
        "          x: xs,\n",
        "          y: ys,\n",
        "          learning_rate: learning_rate_value\n",
        "      })\n",
        "\n",
        "    acc_train = ses.run(accuracy, feed_dict={\n",
        "        x: mnist.train.images,\n",
        "        y: mnist.train.labels\n",
        "    })\n",
        "    if acc_train > acc_train_max:\n",
        "      acc_train_max = acc_train\n",
        "      saver.save(ses, \"./train_model.ckpt\", global_step=max_train_step)\n",
        "\n",
        "    acc_validation = ses.run(accuracy, feed_dict={\n",
        "        x: mnist.validation.images,\n",
        "        y: mnist.validation.labels\n",
        "    })\n",
        "    if acc_validation > acc_validation_max:\n",
        "      acc_validation_max = acc_validation\n",
        "      saver.save(ses, \"./validation_model.ckpt\", global_step=max_train_step)\n",
        "\n",
        "    acc_test = ses.run(accuracy, feed_dict={\n",
        "        x: mnist.test.images,\n",
        "        y: mnist.test.labels\n",
        "    })\n",
        "    if acc_test > acc_test_max:\n",
        "      acc_test_max = acc_test\n",
        "      saver.save(ses, \"./test_model.ckpt\", global_step=max_train_step)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TZzLvEZ-wsYJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "b91720c5-3c47-48c2-8f0a-b753adc79ec0"
      },
      "source": [
        "with tf.Session() as ses:\n",
        "  train_model = saver.restore(ses, \"./train_model.ckpt\" + \"-\" + str(max_train_step))\n",
        "  acc_train = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.train.images,\n",
        "      y: mnist.train.labels\n",
        "  })\n",
        "  acc_validation = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.validation.images,\n",
        "      y: mnist.validation.labels\n",
        "  })\n",
        "  acc_test = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.test.images,\n",
        "      y: mnist.test.labels\n",
        "  })\n",
        "  print(\"using train_model, acc_train:{}, acc_validation:{}, acc_test:{}\".format(acc_train, acc_validation, acc_test))\n"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./train_model.ckpt-1000\n",
            "using train_model, acc_train:0.9673636555671692, acc_validation:0.9639999866485596, acc_test:0.9614999890327454\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WSKLllZsxjG0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "ee2c7084-4014-403e-ad36-3f79379c159f"
      },
      "source": [
        "with tf.Session() as ses:\n",
        "  validation_model = saver.restore(ses, \"./validation_model.ckpt\" + \"-\" + str(max_train_step))\n",
        "  acc_train = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.train.images,\n",
        "      y: mnist.train.labels\n",
        "  })\n",
        "  acc_validation = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.validation.images,\n",
        "      y: mnist.validation.labels\n",
        "  })\n",
        "  acc_test = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.test.images,\n",
        "      y: mnist.test.labels\n",
        "  })\n",
        "  print(\"using validation_model, acc_train:{}, acc_validation:{}, acc_test:{}\".format(acc_train, acc_validation, acc_test))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./validation_model.ckpt-1000\n",
            "using validation_model, acc_train:0.9642181992530823, acc_validation:0.9660000205039978, acc_test:0.958299994468689\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h_xS2sy9yLL-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "58af7898-c305-4541-a4cc-7e473e7f12b9"
      },
      "source": [
        "with tf.Session() as ses:\n",
        "  test_model = saver.restore(ses, \"./test_model.ckpt\" + \"-\" + str(max_train_step))\n",
        "  acc_train = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.train.images,\n",
        "      y: mnist.train.labels\n",
        "  })\n",
        "  acc_validation = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.validation.images,\n",
        "      y: mnist.validation.labels\n",
        "  })\n",
        "  acc_test = ses.run(accuracy, feed_dict={\n",
        "      x: mnist.test.images,\n",
        "      y: mnist.test.labels\n",
        "  })\n",
        "  print(\"using test_model, acc_train:{}, acc_validation:{}, acc_test:{}\".format(acc_train, acc_validation, acc_test))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Restoring parameters from ./test_model.ckpt-1000\n",
            "using test_model, acc_train:0.9654545187950134, acc_validation:0.9599999785423279, acc_test:0.9624000191688538\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}